{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "np.random.seed(12345)\n",
    "np.set_printoptions(precision=4, suppress=True)\n",
    "pd.options.display.max_rows = 25\n",
    "pd.options.display.max_columns = 20\n",
    "pd.options.display.max_colwidth = 82\n",
    "plt.rc(\"figure\", figsize=(10, 6))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:05.173960Z",
     "end_time": "2024-04-18T22:08:27.740603Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 7.1 处理缺失数据"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "0     aardvark\n1    artichoke\n2          NaN\n3         None\n4      avocado\ndtype: object"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_data = pd.Series(['aardvark', 'artichoke', np.nan, None, 'avocado'])\n",
    "string_data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.740603Z",
     "end_time": "2024-04-18T22:08:27.756519Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "0    False\n1    False\n2     True\n3     True\n4    False\ndtype: bool"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_data.isna()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.761042Z",
     "end_time": "2024-04-18T22:08:27.815569Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 过滤缺失数据"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    3.5\n3    NaN\n4    7.0\ndtype: float64"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from numpy import nan as NA\n",
    "\n",
    "data = pd.Series([1, NA, 3.5, NA, 7])\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.780351Z",
     "end_time": "2024-04-18T22:08:27.819425Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "0    1.0\n2    3.5\n4    7.0\ndtype: float64"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.799807Z",
     "end_time": "2024-04-18T22:08:27.819923Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "0    1.0\n2    3.5\n4    7.0\ndtype: float64"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data.notnull()]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.819395Z",
     "end_time": "2024-04-18T22:08:27.836086Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n2  NaN  NaN  NaN\n3  NaN  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame([[1., 6.5, 3.], [1., np.nan, np.nan], [np.nan, np.nan, np.nan], [np.nan, 6.5, 3.]])\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.839706Z",
     "end_time": "2024-04-18T22:08:27.904088Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.860013Z",
     "end_time": "2024-04-18T22:08:27.904088Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n3  NaN  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(how=\"all\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.881717Z",
     "end_time": "2024-04-18T22:08:27.937018Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "     0    1    2   4\n0  1.0  6.5  3.0 NaN\n1  1.0  NaN  NaN NaN\n2  NaN  NaN  NaN NaN\n3  NaN  6.5  3.0 NaN",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[4] = NA\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.904088Z",
     "end_time": "2024-04-18T22:08:28.025998Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "     0    1    2\n0  1.0  6.5  3.0\n1  1.0  NaN  NaN\n2  NaN  NaN  NaN\n3  NaN  6.5  3.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1.0</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>6.5</td>\n      <td>3.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(axis=1, how='all')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.920814Z",
     "end_time": "2024-04-18T22:08:28.081247Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.478943 -0.519439\n1 -0.555730  1.965781  1.393406\n2  0.092908  0.281746  0.769023\n3  1.246435  1.007189 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.478943</td>\n      <td>-0.519439</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>1.965781</td>\n      <td>1.393406</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.281746</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>1.007189</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(np.random.standard_normal((7, 3)))\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.937018Z",
     "end_time": "2024-04-18T22:08:28.086160Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708       NaN       NaN\n1 -0.555730       NaN       NaN\n2  0.092908       NaN  0.769023\n3  1.246435       NaN -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>NaN</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>NaN</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[:4, 1] = NA\n",
    "df.iloc[:2, 2] = NA\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.953292Z",
     "end_time": "2024-04-18T22:08:28.086160Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:27.983965Z",
     "end_time": "2024-04-18T22:08:28.086160Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n2  0.092908       NaN  0.769023\n3  1.246435       NaN -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>NaN</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>NaN</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna(thresh=2)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.001677Z",
     "end_time": "2024-04-18T22:08:28.086160Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 填充缺失数据"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.000000  0.000000\n1 -0.555730  0.000000  0.000000\n2  0.092908  0.000000  0.769023\n3  1.246435  0.000000 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.000000</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>0.000000</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(0)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.028794Z",
     "end_time": "2024-04-18T22:08:28.163321Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.500000  0.000000\n1 -0.555730  0.500000  0.000000\n2  0.092908  0.500000  0.769023\n3  1.246435  0.500000 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.500000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>0.500000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.500000</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>0.500000</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna({1: 0.5, 2: 0})"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.057974Z",
     "end_time": "2024-04-18T22:08:28.205926Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n0 -0.204708  0.000000  0.000000\n1 -0.555730  0.000000  0.000000\n2  0.092908  0.000000  0.769023\n3  1.246435  0.000000 -1.296221\n4  0.274992  0.228913  1.352917\n5  0.886429 -2.001637 -0.371843\n6  1.669025 -0.438570 -0.539741",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-0.204708</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.555730</td>\n      <td>0.000000</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.092908</td>\n      <td>0.000000</td>\n      <td>0.769023</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1.246435</td>\n      <td>0.000000</td>\n      <td>-1.296221</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.274992</td>\n      <td>0.228913</td>\n      <td>1.352917</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.886429</td>\n      <td>-2.001637</td>\n      <td>-0.371843</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>1.669025</td>\n      <td>-0.438570</td>\n      <td>-0.539741</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(0, inplace=True)\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.085663Z",
     "end_time": "2024-04-18T22:08:28.311731Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "          0         1         2\n0  0.476985  3.248944 -1.021228\n1 -0.577087  0.124121  0.302614\n2  0.523772       NaN  1.343810\n3 -0.713544       NaN -2.370232\n4 -1.860761       NaN       NaN\n5 -1.265934       NaN       NaN",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.476985</td>\n      <td>3.248944</td>\n      <td>-1.021228</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.577087</td>\n      <td>0.124121</td>\n      <td>0.302614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.523772</td>\n      <td>NaN</td>\n      <td>1.343810</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.713544</td>\n      <td>NaN</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.860761</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-1.265934</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(np.random.randn(6, 3))\n",
    "df.iloc[2:, 1] = NA\n",
    "df.iloc[4:, 2] = NA\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.106077Z",
     "end_time": "2024-04-18T22:08:28.311731Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\58248\\AppData\\Local\\Temp\\ipykernel_3976\\253117906.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n",
      "  df.fillna(method='ffill')\n"
     ]
    },
    {
     "data": {
      "text/plain": "          0         1         2\n0  0.476985  3.248944 -1.021228\n1 -0.577087  0.124121  0.302614\n2  0.523772  0.124121  1.343810\n3 -0.713544  0.124121 -2.370232\n4 -1.860761  0.124121 -2.370232\n5 -1.265934  0.124121 -2.370232",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.476985</td>\n      <td>3.248944</td>\n      <td>-1.021228</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.577087</td>\n      <td>0.124121</td>\n      <td>0.302614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.523772</td>\n      <td>0.124121</td>\n      <td>1.343810</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.713544</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.860761</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-1.265934</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(method='ffill')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.123391Z",
     "end_time": "2024-04-18T22:08:28.333771Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\58248\\AppData\\Local\\Temp\\ipykernel_3976\\2454611106.py:1: FutureWarning: DataFrame.fillna with 'method' is deprecated and will raise in a future version. Use obj.ffill() or obj.bfill() instead.\n",
      "  df.fillna(method='ffill', limit=2)\n"
     ]
    },
    {
     "data": {
      "text/plain": "          0         1         2\n0  0.476985  3.248944 -1.021228\n1 -0.577087  0.124121  0.302614\n2  0.523772  0.124121  1.343810\n3 -0.713544  0.124121 -2.370232\n4 -1.860761       NaN -2.370232\n5 -1.265934       NaN -2.370232",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.476985</td>\n      <td>3.248944</td>\n      <td>-1.021228</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-0.577087</td>\n      <td>0.124121</td>\n      <td>0.302614</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.523772</td>\n      <td>0.124121</td>\n      <td>1.343810</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.713544</td>\n      <td>0.124121</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.860761</td>\n      <td>NaN</td>\n      <td>-2.370232</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>-1.265934</td>\n      <td>NaN</td>\n      <td>-2.370232</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.fillna(method='ffill', limit=2)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.143432Z",
     "end_time": "2024-04-18T22:08:28.361877Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "0    1.000000\n1    3.833333\n2    3.500000\n3    3.833333\n4    7.000000\ndtype: float64"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.Series([1., NA, 3.5, NA, 7])\n",
    "data.fillna(data.mean())"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.163321Z",
     "end_time": "2024-04-18T22:08:28.381753Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 7.2 数据转换"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 移除重复数据"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "    k1  k2\n0  one   1\n1  two   1\n2  one   2\n3  two   3\n4  one   3\n5  two   4\n6  two   4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame({\"k1\": [\"one\", \"two\"] * 3 + [\"two\"],\n",
    "                     \"k2\": [1, 1, 2, 3, 3, 4, 4]})\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.183590Z",
     "end_time": "2024-04-18T22:08:28.381753Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "0    False\n1    False\n2    False\n3    False\n4    False\n5    False\n6     True\ndtype: bool"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.duplicated()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.209276Z",
     "end_time": "2024-04-18T22:08:28.381753Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "data": {
      "text/plain": "    k1  k2\n0  one   1\n1  two   1\n2  one   2\n3  two   3\n4  one   3\n5  two   4",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.230504Z",
     "end_time": "2024-04-18T22:08:28.401816Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "    k1  k2  v1\n0  one   1   0\n1  two   1   1\n2  one   2   2\n3  two   3   3\n4  one   3   4\n5  two   4   5\n6  two   4   6",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>two</td>\n      <td>4</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['v1'] = range(7)\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.248417Z",
     "end_time": "2024-04-18T22:08:28.536796Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "data": {
      "text/plain": "    k1  k2  v1\n0  one   1   0\n1  two   1   1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates(['k1'])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.282256Z",
     "end_time": "2024-04-18T22:08:28.583702Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "data": {
      "text/plain": "    k1  k2  v1\n0  one   1   0\n1  two   1   1\n2  one   2   2\n3  two   3   3\n4  one   3   4\n6  two   4   6",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>k1</th>\n      <th>k2</th>\n      <th>v1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>one</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>two</td>\n      <td>1</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>one</td>\n      <td>2</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>two</td>\n      <td>3</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>one</td>\n      <td>3</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>two</td>\n      <td>4</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.drop_duplicates(['k1', 'k2'], keep='last')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.311731Z",
     "end_time": "2024-04-18T22:08:28.583702Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 利用函数或映射进行数据转换"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [
    {
     "data": {
      "text/plain": "          food  ounces\n0        bacon     4.0\n1  pulled pork     3.0\n2        bacon    12.0\n3     Pastrami     6.0\n4  corned beef     7.5\n5        Bacon     8.0\n6     pastrami     3.0\n7    honey ham     5.0\n8     nova lox     6.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>food</th>\n      <th>ounces</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bacon</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>pulled pork</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bacon</td>\n      <td>12.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Pastrami</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>corned beef</td>\n      <td>7.5</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Bacon</td>\n      <td>8.0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>pastrami</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>honey ham</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>nova lox</td>\n      <td>6.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame(\n",
    "    {\"food\": [\"bacon\", \"pulled pork\", \"bacon\", \"Pastrami\", \"corned beef\", \"Bacon\", \"pastrami\", \"honey ham\", \"nova lox\"],\n",
    "     \"ounces\": [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.330921Z",
     "end_time": "2024-04-18T22:08:28.712537Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "meat_to_animal = {\n",
    "    \"bacon\": \"pig\",\n",
    "    \"pulled pork\": \"pig\",\n",
    "    \"pastrami\": \"cow\",\n",
    "    \"corned beef\": \"cow\",\n",
    "    \"honey ham\": \"pig\",\n",
    "    \"nova lox\": \"salmon\"\n",
    "}"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.354879Z",
     "end_time": "2024-04-18T22:08:28.740413Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "data": {
      "text/plain": "0          bacon\n1    pulled pork\n2          bacon\n3       pastrami\n4    corned beef\n5          bacon\n6       pastrami\n7      honey ham\n8       nova lox\nName: food, dtype: object"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lowercased = data['food'].str.lower()\n",
    "lowercased"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.375617Z",
     "end_time": "2024-04-18T22:08:28.740413Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [
    {
     "data": {
      "text/plain": "          food  ounces  animal\n0        bacon     4.0     pig\n1  pulled pork     3.0     pig\n2        bacon    12.0     pig\n3     Pastrami     6.0     cow\n4  corned beef     7.5     cow\n5        Bacon     8.0     pig\n6     pastrami     3.0     cow\n7    honey ham     5.0     pig\n8     nova lox     6.0  salmon",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>food</th>\n      <th>ounces</th>\n      <th>animal</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>bacon</td>\n      <td>4.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>pulled pork</td>\n      <td>3.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>bacon</td>\n      <td>12.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Pastrami</td>\n      <td>6.0</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>corned beef</td>\n      <td>7.5</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Bacon</td>\n      <td>8.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>pastrami</td>\n      <td>3.0</td>\n      <td>cow</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>honey ham</td>\n      <td>5.0</td>\n      <td>pig</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>nova lox</td>\n      <td>6.0</td>\n      <td>salmon</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"animal\"] = lowercased.map(meat_to_animal)\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.395169Z",
     "end_time": "2024-04-18T22:08:28.788275Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "data": {
      "text/plain": "0       pig\n1       pig\n2       pig\n3       cow\n4       cow\n5       pig\n6       cow\n7       pig\n8    salmon\nName: food, dtype: object"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"food\"].map(lambda x: meat_to_animal[x.lower()])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.419356Z",
     "end_time": "2024-04-18T22:08:28.788275Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 替换值"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "data": {
      "text/plain": "0       1.0\n1    -999.0\n2       2.0\n3    -999.0\n4   -1000.0\n5       3.0\ndtype: float64"
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.Series([1., -999., 2., -999., -1000., 3.])\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.435463Z",
     "end_time": "2024-04-18T22:08:28.807026Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "data": {
      "text/plain": "0       1.0\n1       NaN\n2       2.0\n3       NaN\n4   -1000.0\n5       3.0\ndtype: float64"
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace(-999, NA)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.452177Z",
     "end_time": "2024-04-18T22:08:28.862287Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [
    {
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    NaN\n5    3.0\ndtype: float64"
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace([-999, -1000], NA)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.478445Z",
     "end_time": "2024-04-18T22:08:28.863169Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [
    {
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    0.0\n5    3.0\ndtype: float64"
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace([-999, -1000], [NA, 0])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.498728Z",
     "end_time": "2024-04-18T22:08:28.873662Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [
    {
     "data": {
      "text/plain": "0    1.0\n1    NaN\n2    2.0\n3    NaN\n4    0.0\n5    3.0\ndtype: float64"
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace({-999: NA, -1000: 0})"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.515454Z",
     "end_time": "2024-04-18T22:08:28.895664Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 重命名轴索引"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "data": {
      "text/plain": "          one  two  three  four\nOhio        0    1      2     3\nColorado    4    5      6     7\nNew York    8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Ohio</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>Colorado</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>New York</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame(np.arange(12).reshape((3, 4)),\n",
    "                    index=[\"Ohio\", \"Colorado\", \"New York\"],\n",
    "                    columns=[\"one\", \"two\", \"three\", \"four\"])\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.540824Z",
     "end_time": "2024-04-18T22:08:28.963711Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [
    {
     "data": {
      "text/plain": "      one  two  three  four\nOHIO    0    1      2     3\nCOLO    4    5      6     7\nNEW     8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>OHIO</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transform = lambda x: x[:4].upper()\n",
    "data.index = data.index.map(transform)\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.557055Z",
     "end_time": "2024-04-18T22:08:28.963711Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [
    {
     "data": {
      "text/plain": "      ONE  TWO  THREE  FOUR\nOhio    0    1      2     3\nColo    4    5      6     7\nNew     8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ONE</th>\n      <th>TWO</th>\n      <th>THREE</th>\n      <th>FOUR</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Ohio</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>Colo</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>New</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.rename(index=str.title, columns=str.upper)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.583702Z",
     "end_time": "2024-04-18T22:08:29.009688Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "outputs": [
    {
     "data": {
      "text/plain": "         one  two  peekaboo  four\nINDIANA    0    1         2     3\nCOLO       4    5         6     7\nNEW        8    9        10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>peekaboo</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>INDIANA</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.rename(index={\"OHIO\": \"INDIANA\"},\n",
    "            columns={\"three\": \"peekaboo\"})"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.602173Z",
     "end_time": "2024-04-18T22:08:29.119306Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [
    {
     "data": {
      "text/plain": "         one  two  three  four\nINDIANA    0    1      2     3\nCOLO       4    5      6     7\nNEW        8    9     10    11",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>one</th>\n      <th>two</th>\n      <th>three</th>\n      <th>four</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>INDIANA</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>COLO</th>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>NEW</th>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.rename(index={'OHIO': 'INDIANA'}, inplace=True)\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.622750Z",
     "end_time": "2024-04-18T22:08:29.178062Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 离散化和面元划分"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [
    {
     "data": {
      "text/plain": "[20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]"
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]\n",
    "ages"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.642736Z",
     "end_time": "2024-04-18T22:08:29.283244Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [
    {
     "data": {
      "text/plain": "[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]\nLength: 12\nCategories (4, interval[int64, right]): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]"
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bins = [18, 25, 35, 60, 100]\n",
    "cats = pd.cut(ages, bins)\n",
    "cats"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.667488Z",
     "end_time": "2024-04-18T22:08:29.289207Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [
    {
     "data": {
      "text/plain": "array([0, 0, 0, 1, 0, 0, 2, 1, 3, 2, 2, 1], dtype=int8)"
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cats.codes"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.680814Z",
     "end_time": "2024-04-18T22:08:29.339353Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [
    {
     "data": {
      "text/plain": "IntervalIndex([(18, 25], (25, 35], (35, 60], (60, 100]], dtype='interval[int64, right]')"
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cats.categories"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.707348Z",
     "end_time": "2024-04-18T22:08:29.339353Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\58248\\AppData\\Local\\Temp\\ipykernel_3976\\3791148250.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(cats)\n"
     ]
    },
    {
     "data": {
      "text/plain": "(18, 25]     5\n(25, 35]     3\n(35, 60]     3\n(60, 100]    1\nName: count, dtype: int64"
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(cats)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.729255Z",
     "end_time": "2024-04-18T22:08:29.509417Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "outputs": [
    {
     "data": {
      "text/plain": "[[18, 26), [18, 26), [18, 26), [26, 36), [18, 26), ..., [26, 36), [61, 100), [36, 61), [36, 61), [26, 36)]\nLength: 12\nCategories (4, interval[int64, left]): [[18, 26) < [26, 36) < [36, 61) < [61, 100)]"
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.cut(ages, [18, 26, 36, 61, 100], right=False)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.752854Z",
     "end_time": "2024-04-18T22:08:29.509417Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "outputs": [
    {
     "data": {
      "text/plain": "['Youth', 'Youth', 'Youth', 'YoungAdult', 'Youth', ..., 'YoungAdult', 'Senior', 'MiddleAged', 'MiddleAged', 'YoungAdult']\nLength: 12\nCategories (4, object): ['Youth' < 'YoungAdult' < 'MiddleAged' < 'Senior']"
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_names = [\"Youth\", \"YoungAdult\", \"MiddleAged\", \"Senior\"]\n",
    "pd.cut(ages, bins, labels=group_names)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.781712Z",
     "end_time": "2024-04-18T22:08:29.509417Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "outputs": [
    {
     "data": {
      "text/plain": "[(0.34, 0.55], (0.34, 0.55], (0.76, 0.97], (0.76, 0.97], (0.34, 0.55], ..., (0.34, 0.55], (0.34, 0.55], (0.55, 0.76], (0.34, 0.55], (0.12, 0.34]]\nLength: 20\nCategories (4, interval[float64, right]): [(0.12, 0.34] < (0.34, 0.55] < (0.55, 0.76] < (0.76, 0.97]]"
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.random.rand(20)\n",
    "pd.cut(data, 4, precision=2)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.806035Z",
     "end_time": "2024-04-18T22:08:29.509417Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "outputs": [
    {
     "data": {
      "text/plain": "[(-0.026, 0.62], (0.62, 3.93], (-0.68, -0.026], (0.62, 3.93], (-0.026, 0.62], ..., (-0.68, -0.026], (-0.68, -0.026], (-2.96, -0.68], (0.62, 3.93], (-0.68, -0.026]]\nLength: 1000\nCategories (4, interval[float64, right]): [(-2.96, -0.68] < (-0.68, -0.026] < (-0.026, 0.62] < (0.62, 3.93]]"
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.random.standard_normal(1000)\n",
    "cats = pd.qcut(data, 4, precision=2)\n",
    "cats"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.824714Z",
     "end_time": "2024-04-18T22:08:29.509417Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\58248\\AppData\\Local\\Temp\\ipykernel_3976\\3791148250.py:1: FutureWarning: pandas.value_counts is deprecated and will be removed in a future version. Use pd.Series(obj).value_counts() instead.\n",
      "  pd.value_counts(cats)\n"
     ]
    },
    {
     "data": {
      "text/plain": "(-2.96, -0.68]     250\n(-0.68, -0.026]    250\n(-0.026, 0.62]     250\n(0.62, 3.93]       250\nName: count, dtype: int64"
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(cats)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.849623Z",
     "end_time": "2024-04-18T22:08:29.586286Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "outputs": [
    {
     "data": {
      "text/plain": "(-2.9499999999999997, -1.187]    100\n(-1.187, -0.0265]                400\n(-0.0265, 1.286]                 400\n(1.286, 3.928]                   100\nName: count, dtype: int64"
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.qcut(data, [0, 0.1, 0.5, 0.9, 1.]).value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.866652Z",
     "end_time": "2024-04-18T22:08:29.586286Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 检测和过滤异常值"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "outputs": [
    {
     "data": {
      "text/plain": "                 0            1            2            3\ncount  1000.000000  1000.000000  1000.000000  1000.000000\nmean      0.049091     0.026112    -0.002544    -0.051827\nstd       0.996947     1.007458     0.995232     0.998311\nmin      -3.645860    -3.184377    -3.745356    -3.428254\n25%      -0.599807    -0.612162    -0.687373    -0.747478\n50%       0.047101    -0.013609    -0.022158    -0.088274\n75%       0.756646     0.695298     0.699046     0.623331\nmax       2.653656     3.525865     2.735527     3.366626",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>0.049091</td>\n      <td>0.026112</td>\n      <td>-0.002544</td>\n      <td>-0.051827</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>0.996947</td>\n      <td>1.007458</td>\n      <td>0.995232</td>\n      <td>0.998311</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>-3.645860</td>\n      <td>-3.184377</td>\n      <td>-3.745356</td>\n      <td>-3.428254</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>-0.599807</td>\n      <td>-0.612162</td>\n      <td>-0.687373</td>\n      <td>-0.747478</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>0.047101</td>\n      <td>-0.013609</td>\n      <td>-0.022158</td>\n      <td>-0.088274</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>0.756646</td>\n      <td>0.695298</td>\n      <td>0.699046</td>\n      <td>0.623331</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>2.653656</td>\n      <td>3.525865</td>\n      <td>2.735527</td>\n      <td>3.366626</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.DataFrame(np.random.standard_normal((1000, 4)))\n",
    "data.describe()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.889952Z",
     "end_time": "2024-04-18T22:08:29.586286Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [
    {
     "data": {
      "text/plain": "41    -3.399312\n136   -3.745356\nName: 2, dtype: float64"
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col = data[2]\n",
    "col[np.abs(col) > 3]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.917246Z",
     "end_time": "2024-04-18T22:08:29.586286Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "outputs": [
    {
     "data": {
      "text/plain": "            0         1         2         3\n41   0.457246 -0.025907 -3.399312 -0.974657\n60   1.951312  3.260383  0.963301  1.201206\n136  0.508391 -0.196713 -3.745356 -1.520113\n235 -0.242459 -3.056990  1.918403 -0.578828\n258  0.682841  0.326045  0.425384 -3.428254\n322  1.179227 -3.184377  1.369891 -1.074833\n544 -3.548824  1.553205 -2.186301  1.277104\n635 -0.578093  0.193299  1.397822  3.366626\n782 -0.207434  3.525865  0.283070  0.544635\n803 -3.645860  0.255475 -0.549574 -1.907459",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>41</th>\n      <td>0.457246</td>\n      <td>-0.025907</td>\n      <td>-3.399312</td>\n      <td>-0.974657</td>\n    </tr>\n    <tr>\n      <th>60</th>\n      <td>1.951312</td>\n      <td>3.260383</td>\n      <td>0.963301</td>\n      <td>1.201206</td>\n    </tr>\n    <tr>\n      <th>136</th>\n      <td>0.508391</td>\n      <td>-0.196713</td>\n      <td>-3.745356</td>\n      <td>-1.520113</td>\n    </tr>\n    <tr>\n      <th>235</th>\n      <td>-0.242459</td>\n      <td>-3.056990</td>\n      <td>1.918403</td>\n      <td>-0.578828</td>\n    </tr>\n    <tr>\n      <th>258</th>\n      <td>0.682841</td>\n      <td>0.326045</td>\n      <td>0.425384</td>\n      <td>-3.428254</td>\n    </tr>\n    <tr>\n      <th>322</th>\n      <td>1.179227</td>\n      <td>-3.184377</td>\n      <td>1.369891</td>\n      <td>-1.074833</td>\n    </tr>\n    <tr>\n      <th>544</th>\n      <td>-3.548824</td>\n      <td>1.553205</td>\n      <td>-2.186301</td>\n      <td>1.277104</td>\n    </tr>\n    <tr>\n      <th>635</th>\n      <td>-0.578093</td>\n      <td>0.193299</td>\n      <td>1.397822</td>\n      <td>3.366626</td>\n    </tr>\n    <tr>\n      <th>782</th>\n      <td>-0.207434</td>\n      <td>3.525865</td>\n      <td>0.283070</td>\n      <td>0.544635</td>\n    </tr>\n    <tr>\n      <th>803</th>\n      <td>-3.645860</td>\n      <td>0.255475</td>\n      <td>-0.549574</td>\n      <td>-1.907459</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[(np.abs(data) > 3).any(axis=1)]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.937698Z",
     "end_time": "2024-04-18T22:08:29.586286Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "outputs": [
    {
     "data": {
      "text/plain": "                 0            1            2            3\ncount  1000.000000  1000.000000  1000.000000  1000.000000\nmean      0.050286     0.025567    -0.001399    -0.051765\nstd       0.992920     1.004214     0.991414     0.995761\nmin      -3.000000    -3.000000    -3.000000    -3.000000\n25%      -0.599807    -0.612162    -0.687373    -0.747478\n50%       0.047101    -0.013609    -0.022158    -0.088274\n75%       0.756646     0.695298     0.699046     0.623331\nmax       2.653656     3.000000     2.735527     3.000000",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n      <td>1000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>0.050286</td>\n      <td>0.025567</td>\n      <td>-0.001399</td>\n      <td>-0.051765</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>0.992920</td>\n      <td>1.004214</td>\n      <td>0.991414</td>\n      <td>0.995761</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n      <td>-3.000000</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>-0.599807</td>\n      <td>-0.612162</td>\n      <td>-0.687373</td>\n      <td>-0.747478</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>0.047101</td>\n      <td>-0.013609</td>\n      <td>-0.022158</td>\n      <td>-0.088274</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>0.756646</td>\n      <td>0.695298</td>\n      <td>0.699046</td>\n      <td>0.623331</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>2.653656</td>\n      <td>3.000000</td>\n      <td>2.735527</td>\n      <td>3.000000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data.abs() > 3] = np.sign(data) * 3\n",
    "data.describe()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.958204Z",
     "end_time": "2024-04-18T22:08:29.596238Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "outputs": [
    {
     "data": {
      "text/plain": "     0    1    2    3\n0 -1.0  1.0 -1.0  1.0\n1  1.0 -1.0  1.0 -1.0\n2  1.0  1.0  1.0 -1.0\n3 -1.0 -1.0  1.0 -1.0\n4 -1.0  1.0 -1.0 -1.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-1.0</td>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>-1.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sign(data).head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:28.985051Z",
     "end_time": "2024-04-18T22:08:29.596238Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 排列和随机采样"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "outputs": [
    {
     "data": {
      "text/plain": "    0   1   2   3   4   5   6\n0   0   1   2   3   4   5   6\n1   7   8   9  10  11  12  13\n2  14  15  16  17  18  19  20\n3  21  22  23  24  25  26  27\n4  28  29  30  31  32  33  34",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7</td>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n      <td>12</td>\n      <td>13</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>14</td>\n      <td>15</td>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n      <td>20</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>21</td>\n      <td>22</td>\n      <td>23</td>\n      <td>24</td>\n      <td>25</td>\n      <td>26</td>\n      <td>27</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>28</td>\n      <td>29</td>\n      <td>30</td>\n      <td>31</td>\n      <td>32</td>\n      <td>33</td>\n      <td>34</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(np.arange(5 * 7).reshape((5, 7)))\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.010185Z",
     "end_time": "2024-04-18T22:08:29.617336Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "outputs": [
    {
     "data": {
      "text/plain": "array([3, 1, 4, 2, 0])"
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampler = np.random.permutation(5)\n",
    "sampler"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.031230Z",
     "end_time": "2024-04-18T22:08:29.617336Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "outputs": [
    {
     "data": {
      "text/plain": "    0   1   2   3   4   5   6\n3  21  22  23  24  25  26  27\n1   7   8   9  10  11  12  13\n4  28  29  30  31  32  33  34\n2  14  15  16  17  18  19  20\n0   0   1   2   3   4   5   6",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>3</th>\n      <td>21</td>\n      <td>22</td>\n      <td>23</td>\n      <td>24</td>\n      <td>25</td>\n      <td>26</td>\n      <td>27</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>7</td>\n      <td>8</td>\n      <td>9</td>\n      <td>10</td>\n      <td>11</td>\n      <td>12</td>\n      <td>13</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>28</td>\n      <td>29</td>\n      <td>30</td>\n      <td>31</td>\n      <td>32</td>\n      <td>33</td>\n      <td>34</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>14</td>\n      <td>15</td>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n      <td>20</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n      <td>4</td>\n      <td>5</td>\n      <td>6</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.take(sampler)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.048081Z",
     "end_time": "2024-04-18T22:08:29.617336Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "outputs": [
    {
     "data": {
      "text/plain": "    0   1   2   3   4   5   6\n3  21  22  23  24  25  26  27\n4  28  29  30  31  32  33  34\n2  14  15  16  17  18  19  20",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>3</th>\n      <td>21</td>\n      <td>22</td>\n      <td>23</td>\n      <td>24</td>\n      <td>25</td>\n      <td>26</td>\n      <td>27</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>28</td>\n      <td>29</td>\n      <td>30</td>\n      <td>31</td>\n      <td>32</td>\n      <td>33</td>\n      <td>34</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>14</td>\n      <td>15</td>\n      <td>16</td>\n      <td>17</td>\n      <td>18</td>\n      <td>19</td>\n      <td>20</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(n=3)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.063775Z",
     "end_time": "2024-04-18T22:08:29.617336Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "outputs": [
    {
     "data": {
      "text/plain": "4    4\n1    7\n4    4\n2   -1\n0    5\n3    6\n1    7\n4    4\n0    5\n4    4\ndtype: int64"
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "choices = pd.Series([5, 7, -1, 6, 4])\n",
    "choices.sample(n=10, replace=True)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.077893Z",
     "end_time": "2024-04-18T22:08:29.642254Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 计算指标/哑变量"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "outputs": [
    {
     "data": {
      "text/plain": "  key  data1\n0   b      0\n1   b      1\n2   a      2\n3   c      3\n4   a      4\n5   b      5",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>key</th>\n      <th>data1</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>b</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>a</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>c</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>a</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>b</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"key\": [\"b\", \"b\", \"a\", \"c\", \"a\", \"b\"],\n",
    "                   \"data1\": range(6)})\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.095561Z",
     "end_time": "2024-04-18T22:08:29.642254Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "outputs": [
    {
     "data": {
      "text/plain": "     a    b    c\n0  0.0  1.0  0.0\n1  0.0  1.0  0.0\n2  1.0  0.0  0.0\n3  0.0  0.0  1.0\n4  1.0  0.0  0.0\n5  0.0  1.0  0.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(df[\"key\"], dtype=float)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.110249Z",
     "end_time": "2024-04-18T22:08:29.642254Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "outputs": [
    {
     "data": {
      "text/plain": "   data1  key_a  key_b  key_c\n0      0    0.0    1.0    0.0\n1      1    0.0    1.0    0.0\n2      2    1.0    0.0    0.0\n3      3    0.0    0.0    1.0\n4      4    1.0    0.0    0.0\n5      5    0.0    1.0    0.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>key_a</th>\n      <th>key_b</th>\n      <th>key_c</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies = pd.get_dummies(df[\"key\"], prefix=\"key\", dtype=float)\n",
    "df_with_dummy = df[[\"data1\"]].join(dummies)\n",
    "df_with_dummy"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.127026Z",
     "end_time": "2024-04-18T22:08:29.665049Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "outputs": [
    {
     "data": {
      "text/plain": "   movie_id                               title                        genres\n0         1                    Toy Story (1995)   Animation|Children's|Comedy\n1         2                      Jumanji (1995)  Adventure|Children's|Fantasy\n2         3             Grumpier Old Men (1995)                Comedy|Romance\n3         4            Waiting to Exhale (1995)                  Comedy|Drama\n4         5  Father of the Bride Part II (1995)                        Comedy\n5         6                         Heat (1995)         Action|Crime|Thriller\n6         7                      Sabrina (1995)                Comedy|Romance\n7         8                 Tom and Huck (1995)          Adventure|Children's\n8         9                 Sudden Death (1995)                        Action\n9        10                    GoldenEye (1995)     Action|Adventure|Thriller",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>movie_id</th>\n      <th>title</th>\n      <th>genres</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>Toy Story (1995)</td>\n      <td>Animation|Children's|Comedy</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>Jumanji (1995)</td>\n      <td>Adventure|Children's|Fantasy</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>Grumpier Old Men (1995)</td>\n      <td>Comedy|Romance</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4</td>\n      <td>Waiting to Exhale (1995)</td>\n      <td>Comedy|Drama</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5</td>\n      <td>Father of the Bride Part II (1995)</td>\n      <td>Comedy</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>6</td>\n      <td>Heat (1995)</td>\n      <td>Action|Crime|Thriller</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>7</td>\n      <td>Sabrina (1995)</td>\n      <td>Comedy|Romance</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>8</td>\n      <td>Tom and Huck (1995)</td>\n      <td>Adventure|Children's</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>9</td>\n      <td>Sudden Death (1995)</td>\n      <td>Action</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>10</td>\n      <td>GoldenEye (1995)</td>\n      <td>Action|Adventure|Thriller</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnames = [\"movie_id\", \"title\", \"genres\"]\n",
    "movies = pd.read_table(\"datasets/movielens/movies.dat\", sep=\"::\",\n",
    "                       header=None, names=mnames, engine=\"python\")\n",
    "movies[:10]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.154265Z",
     "end_time": "2024-04-18T22:08:29.665049Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\58248\\AppData\\Local\\Temp\\ipykernel_3976\\2694543120.py:4: FutureWarning: unique with argument that is not not a Series, Index, ExtensionArray, or np.ndarray is deprecated and will raise in a future version.\n",
      "  genres = pd.unique(all_genres)\n"
     ]
    },
    {
     "data": {
      "text/plain": "array(['Animation', \"Children's\", 'Comedy', 'Adventure', 'Fantasy',\n       'Romance', 'Drama', 'Action', 'Crime', 'Thriller', 'Horror',\n       'Sci-Fi', 'Documentary', 'War', 'Musical', 'Mystery', 'Film-Noir',\n       'Western'], dtype=object)"
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_genres = []\n",
    "for x in movies.genres:\n",
    "    all_genres.extend(x.split('|'))\n",
    "genres = pd.unique(all_genres)\n",
    "genres"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.182202Z",
     "end_time": "2024-04-18T22:08:29.692786Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "outputs": [
    {
     "data": {
      "text/plain": "      Animation  Children's  Comedy  Adventure  Fantasy  Romance  Drama  \\\n0           0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n1           0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n2           0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n3           0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n4           0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n...         ...         ...     ...        ...      ...      ...    ...   \n3878        0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n3879        0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n3880        0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n3881        0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n3882        0.0         0.0     0.0        0.0      0.0      0.0    0.0   \n\n      Action  Crime  Thriller  Horror  Sci-Fi  Documentary  War  Musical  \\\n0        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n1        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n2        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n4        0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n...      ...    ...       ...     ...     ...          ...  ...      ...   \n3878     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3879     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3880     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3881     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n3882     0.0    0.0       0.0     0.0     0.0          0.0  0.0      0.0   \n\n      Mystery  Film-Noir  Western  \n0         0.0        0.0      0.0  \n1         0.0        0.0      0.0  \n2         0.0        0.0      0.0  \n3         0.0        0.0      0.0  \n4         0.0        0.0      0.0  \n...       ...        ...      ...  \n3878      0.0        0.0      0.0  \n3879      0.0        0.0      0.0  \n3880      0.0        0.0      0.0  \n3881      0.0        0.0      0.0  \n3882      0.0        0.0      0.0  \n\n[3883 rows x 18 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Animation</th>\n      <th>Children's</th>\n      <th>Comedy</th>\n      <th>Adventure</th>\n      <th>Fantasy</th>\n      <th>Romance</th>\n      <th>Drama</th>\n      <th>Action</th>\n      <th>Crime</th>\n      <th>Thriller</th>\n      <th>Horror</th>\n      <th>Sci-Fi</th>\n      <th>Documentary</th>\n      <th>War</th>\n      <th>Musical</th>\n      <th>Mystery</th>\n      <th>Film-Noir</th>\n      <th>Western</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>3878</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3879</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3880</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3881</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>3882</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>3883 rows × 18 columns</p>\n</div>"
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zero_matrix = np.zeros((len(movies), len(genres)))\n",
    "dummies = pd.DataFrame(zero_matrix, columns=genres)\n",
    "dummies"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.204872Z",
     "end_time": "2024-04-18T22:08:29.722301Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "outputs": [
    {
     "data": {
      "text/plain": "['Animation', \"Children's\", 'Comedy']"
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gen = movies.genres[0]\n",
    "gen.split('|')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.241570Z",
     "end_time": "2024-04-18T22:08:29.767438Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "outputs": [
    {
     "data": {
      "text/plain": "array([0, 1, 2], dtype=int64)"
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies.columns.get_indexer(gen.split('|'))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.263315Z",
     "end_time": "2024-04-18T22:08:29.767438Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 7.3 字符串操作"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 字符串对象方法"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "outputs": [
    {
     "data": {
      "text/plain": "['a', 'b', '  guido']"
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val = \"a,b,  guido\"\n",
    "val.split(\",\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.283743Z",
     "end_time": "2024-04-18T22:08:29.788640Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "outputs": [
    {
     "data": {
      "text/plain": "['a', 'b', 'guido']"
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pieces = [x.strip() for x in val.split(\",\")]\n",
    "pieces"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.309537Z",
     "end_time": "2024-04-18T22:08:29.788640Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "outputs": [
    {
     "data": {
      "text/plain": "'a::b::guido'"
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first, second, third = pieces\n",
    "first + \"::\" + second + \"::\" + third"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.332134Z",
     "end_time": "2024-04-18T22:08:29.788640Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "outputs": [
    {
     "data": {
      "text/plain": "'a::b::guido'"
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"::\".join(pieces)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.356811Z",
     "end_time": "2024-04-18T22:08:29.807951Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "outputs": [
    {
     "data": {
      "text/plain": "True"
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"guido\" in val"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.365971Z",
     "end_time": "2024-04-18T22:08:29.808448Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "outputs": [
    {
     "data": {
      "text/plain": "1"
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.index(\",\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.381904Z",
     "end_time": "2024-04-18T22:08:29.809936Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "outputs": [
    {
     "data": {
      "text/plain": "-1"
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.find(\":\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.391586Z",
     "end_time": "2024-04-18T22:08:29.809936Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "outputs": [
    {
     "data": {
      "text/plain": "2"
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.count(\",\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.402089Z",
     "end_time": "2024-04-18T22:08:29.878382Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "outputs": [
    {
     "data": {
      "text/plain": "'a::b::  guido'"
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.replace(\",\", \"::\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.424589Z",
     "end_time": "2024-04-18T22:08:29.996860Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "outputs": [
    {
     "data": {
      "text/plain": "'ab  guido'"
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.replace(\",\", \"\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.447915Z",
     "end_time": "2024-04-18T22:08:29.996860Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 正则表达式"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "outputs": [
    {
     "data": {
      "text/plain": "['foo', 'bar', 'baz', 'qux']"
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "text = \"foo    bar\\t baz  \\tqux\"\n",
    "re.split(r\"\\s+\", text)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.467656Z",
     "end_time": "2024-04-18T22:08:29.996860Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "outputs": [
    {
     "data": {
      "text/plain": "['foo', 'bar', 'baz', 'qux']"
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex = re.compile(r\"\\s+\")\n",
    "regex.split(text)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.484258Z",
     "end_time": "2024-04-18T22:08:29.996860Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "outputs": [
    {
     "data": {
      "text/plain": "['    ', '\\t ', '  \\t']"
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.507060Z",
     "end_time": "2024-04-18T22:08:29.996860Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "outputs": [
    {
     "data": {
      "text/plain": "['dave@google.com', 'steve@gmail.com', 'rob@gmail.com', 'ryan@yahoo.com']"
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"\"\"Dave dave@google.com\n",
    "Steve steve@gmail.com\n",
    "Rob rob@gmail.com\n",
    "Ryan ryan@yahoo.com\"\"\"\n",
    "pattern = r\"[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\"\n",
    "\n",
    "# re.IGNORECASE makes the regex case insensitive\n",
    "regex = re.compile(pattern, flags=re.IGNORECASE)\n",
    "regex.findall(text)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.529325Z",
     "end_time": "2024-04-18T22:08:30.333639Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "outputs": [
    {
     "data": {
      "text/plain": "'dave@google.com'"
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = regex.search(text)\n",
    "text[m.start():m.end()]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.546889Z",
     "end_time": "2024-04-18T22:08:30.394602Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(regex.match(text))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.567012Z",
     "end_time": "2024-04-18T22:08:30.394602Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dave REDACTED\n",
      "Steve REDACTED\n",
      "Rob REDACTED\n",
      "Ryan REDACTED\n"
     ]
    }
   ],
   "source": [
    "print(regex.sub(\"REDACTED\", text))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.591793Z",
     "end_time": "2024-04-18T22:08:30.394602Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "outputs": [],
   "source": [
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\"\n",
    "regex = re.compile(pattern, flags=re.IGNORECASE)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.610547Z",
     "end_time": "2024-04-18T22:08:30.458507Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "outputs": [
    {
     "data": {
      "text/plain": "('wesm', 'bright', 'net')"
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m = regex.match(\"wesm@bright.net\")\n",
    "m.groups()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.637472Z",
     "end_time": "2024-04-18T22:08:30.479306Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "outputs": [
    {
     "data": {
      "text/plain": "[('dave', 'google', 'com'),\n ('steve', 'gmail', 'com'),\n ('rob', 'gmail', 'com'),\n ('ryan', 'yahoo', 'com')]"
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex.findall(text)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.660172Z",
     "end_time": "2024-04-18T22:08:30.479306Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dave Username: dave, Domain: google, Suffix: com\n",
      "Steve Username: steve, Domain: gmail, Suffix: com\n",
      "Rob Username: rob, Domain: gmail, Suffix: com\n",
      "Ryan Username: ryan, Domain: yahoo, Suffix: com\n"
     ]
    }
   ],
   "source": [
    "print(regex.sub(r\"Username: \\1, Domain: \\2, Suffix: \\3\", text))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.685624Z",
     "end_time": "2024-04-18T22:08:30.479306Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### pandas的矢量化字符串函数"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     dave@google.com\nSteve    steve@gmail.com\nRob        rob@gmail.com\nWes                  NaN\ndtype: object"
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = {\"Dave\": \"dave@google.com\", \"Steve\": \"steve@gmail.com\",\n",
    "        \"Rob\": \"rob@gmail.com\", \"Wes\": NA}\n",
    "data = pd.Series(data)\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.713849Z",
     "end_time": "2024-04-18T22:08:30.479306Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     False\nSteve    False\nRob      False\nWes       True\ndtype: bool"
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.742734Z",
     "end_time": "2024-04-18T22:08:30.479306Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     False\nSteve     True\nRob       True\nWes        NaN\ndtype: object"
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.contains('gmail')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.767438Z",
     "end_time": "2024-04-18T22:08:30.479306Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     dave@google.com\nSteve    steve@gmail.com\nRob        rob@gmail.com\nWes                 <NA>\ndtype: string"
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_as_string_ext = data.astype('string')\n",
    "data_as_string_ext"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.786893Z",
     "end_time": "2024-04-18T22:08:30.490325Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     False\nSteve     True\nRob       True\nWes       <NA>\ndtype: boolean"
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_as_string_ext.str.contains(\"gmail\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.808943Z",
     "end_time": "2024-04-18T22:08:30.554553Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     [(dave, google, com)]\nSteve    [(steve, gmail, com)]\nRob        [(rob, gmail, com)]\nWes                        NaN\ndtype: object"
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern = r\"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\\.([A-Z]{2,4})\"\n",
    "data.str.findall(pattern, flags=re.IGNORECASE)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.830132Z",
     "end_time": "2024-04-18T22:08:30.563305Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     (dave, google, com)\nSteve    (steve, gmail, com)\nRob        (rob, gmail, com)\nWes                      NaN\ndtype: object"
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches = data.str.findall(pattern, flags=re.IGNORECASE).str[0]\n",
    "matches"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.851440Z",
     "end_time": "2024-04-18T22:08:30.563305Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     google\nSteve     gmail\nRob       gmail\nWes         NaN\ndtype: object"
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matches.str.get(1)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.883038Z",
     "end_time": "2024-04-18T22:08:30.575068Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "outputs": [
    {
     "data": {
      "text/plain": "Dave     dave@\nSteve    steve\nRob      rob@g\nWes        NaN\ndtype: object"
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str[:5]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.903800Z",
     "end_time": "2024-04-18T22:08:30.595503Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "outputs": [
    {
     "data": {
      "text/plain": "           0       1    2\nDave    dave  google  com\nSteve  steve   gmail  com\nRob      rob   gmail  com\nWes      NaN     NaN  NaN",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Dave</th>\n      <td>dave</td>\n      <td>google</td>\n      <td>com</td>\n    </tr>\n    <tr>\n      <th>Steve</th>\n      <td>steve</td>\n      <td>gmail</td>\n      <td>com</td>\n    </tr>\n    <tr>\n      <th>Rob</th>\n      <td>rob</td>\n      <td>gmail</td>\n      <td>com</td>\n    </tr>\n    <tr>\n      <th>Wes</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.str.extract(pattern, flags=re.IGNORECASE)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.928292Z",
     "end_time": "2024-04-18T22:08:30.629547Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2024-04-18T22:08:29.952020Z",
     "end_time": "2024-04-18T22:08:30.650322Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
